{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import *\n",
    "from transformers import AdamW\n",
    "import torch.utils.data as Data\n",
    "import collections\n",
    "import os\n",
    "import random\n",
    "import tarfile\n",
    "import torch\n",
    "from torch import nn\n",
    "import torchtext.vocab as Vocab\n",
    "import pickle as pk\n",
    "import copy\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import torch.nn.functional as F\n",
    "from IPython.display import display,HTML\n",
    "import os\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch.nn.utils.rnn import pack_padded_sequence\n",
    "from torch.nn.utils.rnn import pad_packed_sequence\n",
    "from torch.nn.utils.rnn import pack_sequence\n",
    "from torch.nn import CrossEntropyLoss, MSELoss\n",
    "import math\n",
    "device=torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")\n",
    "import argparse\n",
    "import glob\n",
    "import json\n",
    "import logging\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset\n",
    "from torch.utils.data.distributed import DistributedSampler\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import torch.utils.data as Data\n",
    "logger = logging.getLogger(__name__)\n",
    "logging.basicConfig(\n",
    "    format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
    "    datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
    "    level=logging.INFO \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bert 尝试"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### model list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Transformers has a unified API\n",
    "# for 10 transformer architectures and 30 pretrained weights.\n",
    "#          Model          | Tokenizer          | Pretrained weights shortcut\n",
    "MODELS = [(BertModel,       BertTokenizer,       'bert-base-uncased'),\n",
    "          (OpenAIGPTModel,  OpenAIGPTTokenizer,  'openai-gpt'),\n",
    "          (GPT2Model,       GPT2Tokenizer,       'gpt2'),\n",
    "          (CTRLModel,       CTRLTokenizer,       'ctrl'),\n",
    "          (TransfoXLModel,  TransfoXLTokenizer,  'transfo-xl-wt103'),\n",
    "          (XLNetModel,      XLNetTokenizer,      'xlnet-base-cased'),\n",
    "          (XLMModel,        XLMTokenizer,        'xlm-mlm-enfr-1024'),\n",
    "          (DistilBertModel, DistilBertTokenizer, 'distilbert-base-cased'),\n",
    "          (RobertaModel,    RobertaTokenizer,    'roberta-base'),\n",
    "          (XLMRobertaModel, XLMRobertaTokenizer, 'xlm-roberta-base'),\n",
    "         ]\n",
    "bert_dir=\"./bert-pytorch/\"\n",
    "vocab=\"bert-base-uncased-vocab.txt\"\n",
    "config_file=\"bert_config.json\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### load 方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Calling BertTokenizer.from_pretrained() with the path to a single file or url is deprecated\n"
     ]
    }
   ],
   "source": [
    "model=BertModel.from_pretrained(bert_dir,config=os.path.join(bert_dir,config_file),output_hidden_states=True,\n",
    "                                        output_attentions=True)\n",
    "tokenizer=BertTokenizer.from_pretrained(os.path.join(bert_dir,vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### tokenizer使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 9, 768]),\n",
       " tensor([[  101,  2182,  2003,  2070,  3793,  2000,  4372, 16044,   102]]),\n",
       " '[CLS] here is some text to encode [SEP]',\n",
       " ['[CLS]', 'here', 'is', 'some', 'text', 'to', 'en', '##code', '[SEP]'])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids=torch.tensor([tokenizer.encode(\"here is some text to encode\")])\n",
    "model(input_ids)[0].shape,input_ids,tokenizer.decode(input_ids[0]),tokenizer.convert_ids_to_tokens(tokenizer.encode(\"here is some text to encode\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### 各类任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model = BertForMultipleChoice.from_pretrained(bert_dir,config=os.path.join(bert_dir,config))\n",
    "choices = [\"Hello, my dog is cute\", \"Hello, fuck off your mother\",\"Hello, fuck off your mother\"]\n",
    "input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0)  # Batch size 1, 2 choices\n",
    "labels = torch.tensor(2).unsqueeze(0)  # Batch size 1\n",
    "outputs = model(input_ids, labels=labels)\n",
    "loss, classification_scores = outputs[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.3965, -0.3958, -0.3958]], grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classification_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model = BertForTokenClassification.from_pretrained(bert_dir,config=os.path.join(bert_dir,config))\n",
    "input_ids = torch.tensor(tokenizer.encode(\"Hello, my dog is cute\")).unsqueeze(0)  # Batch size 1\n",
    "labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1\n",
    "outputs = model(input_ids, labels=labels)\n",
    "loss, scores = outputs[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model = BertForQuestionAnswering.from_pretrained(bert_dir,config=os.path.join(bert_dir,config))\n",
    "input_ids = torch.tensor(tokenizer.encode(\"Hello, my dog is cute\")).unsqueeze(0)  # Batch size 1\n",
    "start_positions = torch.tensor([1])\n",
    "end_positions = torch.tensor([3])\n",
    "outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions)\n",
    "loss, start_scores, end_scores = outputs[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 16])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.encode_plus([\"Hello, my dog is cute\",\"Hello, my dog is cute\"],max_length=16,pad_to_max_length=True,return_tensors=\"pt\")['input_ids'].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### model存储"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "BERT_MODEL_CLASSES = [BertModel, BertForPreTraining, BertForMaskedLM, BertForNextSentencePrediction,\n",
    "                      BertForSequenceClassification, BertForTokenClassification, BertForQuestionAnswering]\n",
    "for model_class in BERT_MODEL_CLASSES:\n",
    "    print(model_class)\n",
    "    model=model_class.from_pretrained(bert_dir,config=os.path.join(bert_dir,config),output_hidden_states=True,\n",
    "                                        output_attentions=True)\n",
    "    input_ids=torch.tensor([tokenizer.encode(\"Let's see all hidden-states and attentions on this text\")])\n",
    "    all_hidden_states,all_attentions=model(input_ids)[-2:]\n",
    "    model = model_class.from_pretrained(bert_dir,config=os.path.join(bert_dir,config), torchscript=True)\n",
    "    model.save_pretrained('../../model_saved/')  # save\n",
    "#     model = model_class.from_pretrained('./directory/to/save/')  # re-load\n",
    "    tokenizer.save_pretrained('../../model_saved/')  # save\n",
    "#     tokenizer = BertTokenizer.from_pretrained('./directory/to/save/')  # re-load\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 情感分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import collections\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "code_folding": [
     21
    ]
   },
   "outputs": [],
   "source": [
    "class BertMulticlass(BertPreTrainedModel):\n",
    "    def __init__(self, config):\n",
    "        super().__init__(config)\n",
    "        self.num_labels = config.num_labels\n",
    "\n",
    "        self.bert = BertModel(config)\n",
    "        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n",
    "        self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)\n",
    "\n",
    "        self.init_weights()\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        attention_mask=None,\n",
    "        token_type_ids=None,\n",
    "        position_ids=None,\n",
    "        head_mask=None,\n",
    "        inputs_embeds=None,\n",
    "        labels=None,\n",
    "    ):\n",
    "\n",
    "        outputs = self.bert(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            token_type_ids=token_type_ids,\n",
    "            position_ids=position_ids,\n",
    "            head_mask=head_mask,\n",
    "            inputs_embeds=inputs_embeds,\n",
    "        )\n",
    "\n",
    "        pooled_output = outputs[1]\n",
    "\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        logits = self.classifier(pooled_output)\n",
    "\n",
    "        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here\n",
    "\n",
    "        if labels is not None:\n",
    "            if self.num_labels == 1:\n",
    "                #  We are doing regression\n",
    "                loss_fct = MSELoss()\n",
    "                loss = loss_fct(logits.view(-1), labels.view(-1))\n",
    "            else:\n",
    "                loss_fct = nn.BCEWithLogitsLoss()\n",
    "                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1,self.num_labels))\n",
    "            outputs = (loss,) + outputs\n",
    "\n",
    "        return outputs  # (loss), logits, (hidden_states), (attentions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "code_folding": [
     0,
     8
    ]
   },
   "outputs": [],
   "source": [
    "def set_seed(args):\n",
    "    random.seed(args.seed)\n",
    "    np.random.seed(args.seed)\n",
    "    torch.manual_seed(args.seed)\n",
    "    if args.n_gpu > 0:\n",
    "        torch.cuda.manual_seed_all(args.seed)\n",
    "\n",
    "\n",
    "def train(args, train_dataset,val_dataset, model, tokenizer):\n",
    "#     \"\"\" Train the model \"\"\"\n",
    "#     if args.local_rank in [-1, 0]:\n",
    "#         tb_writer = SummaryWriter()\n",
    "\n",
    "#     args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)\n",
    "    train_sampler = RandomSampler(train_dataset) # if args.local_rank == -1 else DistributedSampler(train_dataset)\n",
    "    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)\n",
    "\n",
    "    if args.max_steps > 0:\n",
    "        t_total = args.max_steps\n",
    "        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1\n",
    "    else:\n",
    "        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs\n",
    "\n",
    "    # Prepare optimizer and schedule (linear warmup and decay)\n",
    "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
    "    optimizer_grouped_parameters = [\n",
    "        {\n",
    "            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
    "            \"weight_decay\": args.weight_decay,\n",
    "        },\n",
    "        {\"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], \"weight_decay\": 0.0},\n",
    "    ]\n",
    "\n",
    "    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)\n",
    "    scheduler = get_linear_schedule_with_warmup(\n",
    "        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total\n",
    "    )\n",
    "\n",
    "    # Check if saved optimizer or scheduler states exist\n",
    "    if os.path.isfile(os.path.join(args.model_name_or_path, \"optimizer.pt\")) and os.path.isfile(\n",
    "        os.path.join(args.model_name_or_path, \"scheduler.pt\")\n",
    "    ):\n",
    "        logger.info(\"  loading optimizer and scheduler...\")\n",
    "        # Load in optimizer and scheduler states\n",
    "        optimizer.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"optimizer.pt\")))\n",
    "        scheduler.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"scheduler.pt\")))\n",
    "    else:\n",
    "        logger.info(\"  No optimizer and scheduler,we build a new one\")        \n",
    "\n",
    "#     if args.fp16:\n",
    "#         try:\n",
    "#             from apex import amp\n",
    "#         except ImportError:\n",
    "#             raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to use fp16 training.\")\n",
    "#         model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)\n",
    "\n",
    "    # multi-gpu training (should be after apex fp16 initialization)\n",
    "#     if args.n_gpu > 1:\n",
    "#         model = torch.nn.DataParallel(model)\n",
    "\n",
    "    # Distributed training (should be after apex fp16 initialization)\n",
    "#     if args.local_rank != -1:\n",
    "#         model = torch.nn.parallel.DistributedDataParallel(\n",
    "#             model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True,\n",
    "#         )\n",
    "\n",
    "    # Train!\n",
    "    logger.info(\"***** Running training *****\")\n",
    "    logger.info(\"  Num examples = %d\", len(train_dataset))\n",
    "    logger.info(\"  Num Epochs = %d\", args.num_train_epochs)\n",
    "#     logger.info(\"  Instantaneous batch size per GPU = %d\", args.per_gpu_train_batch_size)\n",
    "    logger.info(\n",
    "        \"  Total train batch size (w. parallel, distributed & accumulation) = %d\",\n",
    "        args.train_batch_size\n",
    "        * args.gradient_accumulation_steps\n",
    "#         * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),\n",
    "    )\n",
    "    logger.info(\"  Gradient Accumulation steps = %d\", args.gradient_accumulation_steps)\n",
    "    logger.info(\"  Total optimization steps = %d\", t_total)\n",
    "\n",
    "    global_step = 0\n",
    "    epochs_trained = 0\n",
    "    steps_trained_in_current_epoch = 0\n",
    "    # Check if continuing training from a checkpoint\n",
    "    if os.path.exists(args.model_name_or_path):\n",
    "        # set global_step to global_step of last saved checkpoint from model path\n",
    "        try:\n",
    "            global_step = int(args.model_name_or_path.split(\"-\")[-1].split(\"/\")[0])\n",
    "        except ValueError:\n",
    "            global_step = 0\n",
    "        epochs_trained = global_step // (len(train_dataloader) // args.gradient_accumulation_steps)\n",
    "        steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps)\n",
    "\n",
    "        logger.info(\"  Continuing training from checkpoint, will skip to saved global_step\")\n",
    "        logger.info(\"  Continuing training from epoch %d\", epochs_trained)\n",
    "        logger.info(\"  Continuing training from global step %d\", global_step)\n",
    "        logger.info(\"  Will skip the first %d steps in the first epoch\", steps_trained_in_current_epoch)\n",
    "\n",
    "    tr_loss, logging_loss = 0.0, 0.0\n",
    "    model.zero_grad()\n",
    "    train_iterator = tqdm(range(\n",
    "        epochs_trained, int(args.num_train_epochs)), desc=\"Epoch\")\n",
    "    set_seed(args)  # Added here for reproductibility\n",
    "    for _ in train_iterator:\n",
    "        epoch_iterator = tqdm(train_dataloader, desc=\"Iteration\")\n",
    "        for step, batch in enumerate(epoch_iterator):\n",
    "\n",
    "            # Skip past any already trained steps if resuming training\n",
    "            if steps_trained_in_current_epoch > 0:\n",
    "                if  (step + 1) % args.gradient_accumulation_steps == 0: \n",
    "                        steps_trained_in_current_epoch -= 1\n",
    "                continue\n",
    "            print(step)\n",
    "\n",
    "            model.train()\n",
    "            batch = tuple(t.to(args.device) for t in batch)\n",
    "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3]}\n",
    "            inputs[\"token_type_ids\"]=batch[2]\n",
    "#             if args.model_type != \"distilbert\":\n",
    "#                 inputs[\"token_type_ids\"] = (\n",
    "#                     batch[2] if args.model_type in [\"bert\", \"xlnet\", \"albert\"] else None\n",
    "#                 )  # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids\n",
    "            outputs = model(**inputs)\n",
    "            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)\n",
    "\n",
    "            if args.n_gpu > 1:\n",
    "                loss = loss.mean()  # mean() to average on multi-gpu parallel training\n",
    "            if args.gradient_accumulation_steps > 1:\n",
    "                loss = loss / args.gradient_accumulation_steps\n",
    "\n",
    "#             if args.fp16:\n",
    "#                 with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "#                     scaled_loss.backward()\n",
    "#             else:\n",
    "            loss.backward()\n",
    "\n",
    "            tr_loss += loss.item()\n",
    "            if (step + 1) % args.gradient_accumulation_steps == 0:\n",
    "#                 if args.fp16:\n",
    "#                     torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)\n",
    "#                 else:\n",
    "                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)\n",
    "\n",
    "                optimizer.step()\n",
    "                scheduler.step()  # Update learning rate schedule\n",
    "                model.zero_grad()\n",
    "                global_step += 1\n",
    "\n",
    "#                 if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:\n",
    "#                     logs = {}\n",
    "#                     if (\n",
    "#                         args.local_rank == -1 and args.evaluate_during_training\n",
    "#                     ):  # Only evaluate when single GPU otherwise metrics may not average well\n",
    "#                         results = evaluate(args, model, tokenizer)\n",
    "#                         for key, value in results.items():\n",
    "#                             eval_key = \"eval_{}\".format(key)\n",
    "#                             logs[eval_key] = value\n",
    "\n",
    "#                     loss_scalar = (tr_loss - logging_loss) / args.logging_steps\n",
    "#                     learning_rate_scalar = scheduler.get_lr()[0]\n",
    "#                     logs[\"learning_rate\"] = learning_rate_scalar\n",
    "#                     logs[\"loss\"] = loss_scalar\n",
    "#                     logging_loss = tr_loss\n",
    "\n",
    "#                     for key, value in logs.items():\n",
    "#                         tb_writer.add_scalar(key, value, global_step)\n",
    "#                     print(json.dumps({**logs, **{\"step\": global_step}}))\n",
    "\n",
    "                if  args.save_steps > 0 and global_step % args.save_steps == 0:\n",
    "                    # Save model checkpoint\n",
    "                    results = evaluate(args, val_dataset,model, tokenizer)\n",
    "                    output_dir = os.path.join(args.output_dir, \"checkpoint-{}\".format(global_step))\n",
    "                    if not os.path.exists(output_dir):\n",
    "                        os.makedirs(output_dir)\n",
    "                    model_to_save = (\n",
    "                        model.module if hasattr(model, \"module\") else model\n",
    "                    )  # Take care of distributed/parallel training\n",
    "                    model_to_save.save_pretrained(output_dir)\n",
    "                    tokenizer.save_pretrained(output_dir)\n",
    "\n",
    "                    torch.save(args, os.path.join(output_dir, \"training_args.bin\"))\n",
    "                    logger.info(\"Saving model checkpoint to %s\", output_dir)\n",
    "\n",
    "                    torch.save(optimizer.state_dict(), os.path.join(output_dir, \"optimizer.pt\"))\n",
    "                    torch.save(scheduler.state_dict(), os.path.join(output_dir, \"scheduler.pt\"))\n",
    "                    logger.info(\"Saving optimizer and scheduler states to %s\", output_dir)\n",
    "\n",
    "            if args.max_steps > 0 and global_step > args.max_steps:\n",
    "                epoch_iterator.close()\n",
    "                break\n",
    "        if args.max_steps > 0 and global_step > args.max_steps:\n",
    "            train_iterator.close()\n",
    "            break\n",
    "\n",
    "#     if args.local_rank in [-1, 0]:\n",
    "#         tb_writer.close()\n",
    "\n",
    "    return global_step, tr_loss / global_step\n",
    "\n",
    "\n",
    "def evaluate(args, eval_dataset,model, tokenizer, prefix=\"\"):\n",
    "    eval_output_dir = args.output_dir \n",
    "\n",
    "    results = {}\n",
    "#         eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True)\n",
    "\n",
    "    if not os.path.exists(eval_output_dir) :\n",
    "        os.makedirs(eval_output_dir)\n",
    "\n",
    "#         args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)\n",
    "    # Note that DistributedSampler samples randomly\n",
    "    eval_sampler = SequentialSampler(eval_dataset)\n",
    "    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)\n",
    "\n",
    "    # multi-gpu eval\n",
    "#         if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):\n",
    "#             model = torch.nn.DataParallel(model)\n",
    "\n",
    "    # Eval!\n",
    "    logger.info(\"***** Running evaluation {} *****\".format(prefix))\n",
    "    logger.info(\"  Num examples = %d\", len(eval_dataset))\n",
    "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
    "    eval_loss = 0.0\n",
    "    nb_eval_steps = 0\n",
    "    preds = None\n",
    "    out_label_ids = None\n",
    "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
    "        model.eval()\n",
    "        batch = tuple(t.to(args.device) for t in batch)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3]}\n",
    "            inputs[\"token_type_ids\"]=batch[2]\n",
    "#                 if args.model_type != \"distilbert\":\n",
    "#                     inputs[\"token_type_ids\"] = (\n",
    "#                         batch[2] if args.model_type in [\"bert\", \"xlnet\", \"albert\"] else None\n",
    "#                     )  # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids\n",
    "            outputs = model(**inputs)\n",
    "            tmp_eval_loss, logits = outputs[:2]\n",
    "\n",
    "            eval_loss += tmp_eval_loss.mean().item()\n",
    "        nb_eval_steps += 1\n",
    "        if preds is None:\n",
    "            preds = logits.detach().cpu().numpy()\n",
    "            out_label_ids = inputs[\"labels\"].detach().cpu().numpy()\n",
    "        else:\n",
    "            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)\n",
    "            out_label_ids = np.append(out_label_ids, inputs[\"labels\"].detach().cpu().numpy(), axis=0)\n",
    "\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    if args.output_mode == \"classification\":\n",
    "        preds = np.argmax(preds, axis=1)\n",
    "    elif args.output_mode == \"regression\":\n",
    "        preds = np.squeeze(preds)\n",
    "    result = {\"acc\":(preds==out_label_ids).sum()/preds.shape[0]}\n",
    "    results.update(result)\n",
    "\n",
    "    output_eval_file = os.path.join(eval_output_dir, prefix, \"eval_results.txt\")\n",
    "    with open(output_eval_file, \"a\") as writer:\n",
    "        logger.info(\"***** Eval results {} *****\".format(prefix))\n",
    "        for key in sorted(result.keys()):\n",
    "            logger.info(\"  %s = %s\", key, str(result[key]))\n",
    "            writer.write(\"%s = %s\\n\" % (key, str(result[key])))\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "code_folding": [
     0,
     16
    ]
   },
   "outputs": [],
   "source": [
    "def convert_text_to_ids(tokenizer, text, max_len=100):\n",
    "    if isinstance(text,str):\n",
    "        output=tokenizer.encode_plus(text,max_length=max_len,pad_to_max_length=True,return_tensors=\"pt\")\n",
    "        input_ids=output[\"input_ids\"].squeeze(0)\n",
    "        token_type_ids=output[\"token_type_ids\"].squeeze(0)\n",
    "        attention_mask=output[\"attention_mask\"].squeeze(0)\n",
    "    elif isinstance(text,list):\n",
    "        input_ids,token_type_ids,attention_mask=[],[],[]\n",
    "        for e in text:\n",
    "            output=tokenizer.encode_plus(e,max_length=max_len,pad_to_max_length=True,return_tensors=\"pt\")\n",
    "            input_ids.append(output[\"input_ids\"].squeeze(0))\n",
    "            token_type_ids.append(output[\"token_type_ids\"].squeeze(0))\n",
    "            attention_mask.append(output[\"attention_mask\"].squeeze(0))\n",
    "    else:\n",
    "        raise Exception('type error')\n",
    "    return torch.stack(input_ids).long(),torch.stack(token_type_ids).long(),torch.stack(attention_mask).long()        \n",
    "class SentimentDataset(Data.Dataset):\n",
    "    def __init__(self,path,tokenizer,max_len):\n",
    "        self.dataset=pd.read_csv(path,sep=\"\\t\",names=[\"text\",\"label\"]).values\n",
    "        self.tokenizer=tokenizer\n",
    "        text=self.dataset[:,0]\n",
    "        self.label=torch.tensor(self.dataset[:,1].astype(np.int32)).long()\n",
    "        self.input_ids,self.token_type_ids,self.attention_mask=convert_text_to_ids(tokenizer,list(text))\n",
    "    def __len__(self):\n",
    "        return self.dataset.shape[0]\n",
    "    def __getitem__(self,idx):\n",
    "        return self.input_ids[idx],self.attention_mask[idx],self.token_type_ids[idx],self.label[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = \"./test-data/sentiment/\"\n",
    "max_seq_length=128\n",
    "# 加载数据集\n",
    "sentiment_train_set = SentimentDataset(data_path + \"sentiment.train.data\",tokenizer,max_seq_length)\n",
    "sentiment_valid_set = SentimentDataset(data_path + \"sentiment.valid.data\",tokenizer,max_seq_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "ARG=collections.namedtuple('ARG',['train_batch_size',\n",
    " 'eval_batch_size',\n",
    " 'weight_decay',\n",
    " 'learning_rate',\n",
    " 'adam_epsilon',\n",
    " 'num_train_epochs',\n",
    " 'warmup_steps',\n",
    " 'gradient_accumulation_steps',\n",
    " 'save_steps',\n",
    " 'max_grad_norm',\n",
    " 'model_name_or_path',\n",
    " 'output_dir',\n",
    " 'seed',\n",
    " 'device',\n",
    " 'n_gpu',\n",
    " 'max_steps',\n",
    " 'output_mode'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 初次训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "hidden_dropout_prob = 0.3\n",
    "num_labels = 2\n",
    "learning_rate = 5e-5\n",
    "weight_decay = 0\n",
    "epochs = 2\n",
    "batch_size = 16\n",
    "adam_epsilon=1e-8\n",
    "do_lower_case=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:11:31 - INFO - transformers.configuration_utils -   loading configuration file ./bert-pytorch-chinese/bert_config.json\n",
      "03/26/2020 21:11:31 - INFO - transformers.configuration_utils -   Model config BertConfig {\n",
      "  \"_num_labels\": 2,\n",
      "  \"architectures\": null,\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": null,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"do_sample\": false,\n",
      "  \"early_stopping\": false,\n",
      "  \"eos_token_ids\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.3,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": true,\n",
      "  \"output_hidden_states\": true,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": null,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "03/26/2020 21:11:31 - INFO - transformers.modeling_utils -   loading weights file ./bert-pytorch-chinese/pytorch_model.bin\n",
      "03/26/2020 21:11:36 - INFO - transformers.modeling_utils -   Weights of BertFJW not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "03/26/2020 21:11:36 - INFO - transformers.modeling_utils -   Weights from pretrained model not used in BertFJW: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "03/26/2020 21:11:36 - INFO - transformers.tokenization_utils -   Model name './bert-pytorch-chinese/bert-base-chinese-vocab.txt' not found in model shortcut name list (bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese, bert-base-german-cased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking, bert-large-uncased-whole-word-masking-finetuned-squad, bert-large-cased-whole-word-masking-finetuned-squad, bert-base-cased-finetuned-mrpc, bert-base-german-dbmdz-cased, bert-base-german-dbmdz-uncased, bert-base-finnish-cased-v1, bert-base-finnish-uncased-v1, bert-base-dutch-cased). Assuming './bert-pytorch-chinese/bert-base-chinese-vocab.txt' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "03/26/2020 21:11:36 - WARNING - transformers.tokenization_utils -   Calling BertTokenizer.from_pretrained() with the path to a single file or url is deprecated\n",
      "03/26/2020 21:11:36 - INFO - transformers.tokenization_utils -   loading file ./bert-pytorch-chinese/bert-base-chinese-vocab.txt\n"
     ]
    }
   ],
   "source": [
    "device=torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
    "bert_dir=\"./bert-pytorch-chinese/\"\n",
    "vocab=\"bert-base-chinese-vocab.txt\"\n",
    "config_file=\"bert_config.json\"\n",
    "config=BertConfig.from_pretrained(os.path.join(bert_dir,config_file),num_labels=num_labels,output_hidden_states=True,\n",
    "                                        output_attentions=True,hidden_dropout_prob=hidden_dropout_prob)\n",
    "model=BertFJW.from_pretrained(bert_dir,config=config)\n",
    "tokenizer=BertTokenizer.from_pretrained(os.path.join(bert_dir,vocab),do_lower_case=do_lower_case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "args=ARG(train_batch_size=batch_size,eval_batch_size=batch_size,weight_decay=weight_decay,learning_rate=learning_rate,\n",
    "         adam_epsilon=adam_epsilon,num_train_epochs=epochs,warmup_steps=0,gradient_accumulation_steps=1,save_steps=500,\n",
    "         max_grad_norm=1.0,model_name_or_path=\"./outputs/\",output_dir=\"./outputs/\",seed=42,device=device,n_gpu=0,\n",
    "        max_steps=0,output_mode=\"classification\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:11:41 - INFO - __main__ -     No optimizer and scheduler,we build a new one\n",
      "03/26/2020 21:11:41 - INFO - __main__ -   ***** Running training *****\n",
      "03/26/2020 21:11:41 - INFO - __main__ -     Num examples = 16839\n",
      "03/26/2020 21:11:41 - INFO - __main__ -     Num Epochs = 2\n",
      "03/26/2020 21:11:41 - INFO - __main__ -     Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "03/26/2020 21:11:41 - INFO - __main__ -     Gradient Accumulation steps = 1\n",
      "03/26/2020 21:11:41 - INFO - __main__ -     Total optimization steps = 2106\n",
      "03/26/2020 21:11:41 - INFO - __main__ -     Continuing training from checkpoint, will skip to saved global_step\n",
      "03/26/2020 21:11:41 - INFO - __main__ -     Continuing training from epoch 0\n",
      "03/26/2020 21:11:41 - INFO - __main__ -     Continuing training from global step 0\n",
      "03/26/2020 21:11:41 - INFO - __main__ -     Will skip the first 0 steps in the first epoch\n",
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:102: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a307f1f77eb4d3d8fe7b805cf3eae1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:105: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0eed6ac0ebfa4bb5a99373ca69b8a1aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=1053, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n",
      "210\n",
      "211\n",
      "212\n",
      "213\n",
      "214\n",
      "215\n",
      "216\n",
      "217\n",
      "218\n",
      "219\n",
      "220\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "234\n",
      "235\n",
      "236\n",
      "237\n",
      "238\n",
      "239\n",
      "240\n",
      "241\n",
      "242\n",
      "243\n",
      "244\n",
      "245\n",
      "246\n",
      "247\n",
      "248\n",
      "249\n",
      "250\n",
      "251\n",
      "252\n",
      "253\n",
      "254\n",
      "255\n",
      "256\n",
      "257\n",
      "258\n",
      "259\n",
      "260\n",
      "261\n",
      "262\n",
      "263\n",
      "264\n",
      "265\n",
      "266\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "287\n",
      "288\n",
      "289\n",
      "290\n",
      "291\n",
      "292\n",
      "293\n",
      "294\n",
      "295\n",
      "296\n",
      "297\n",
      "298\n",
      "299\n",
      "300\n",
      "301\n",
      "302\n",
      "303\n",
      "304\n",
      "305\n",
      "306\n",
      "307\n",
      "308\n",
      "309\n",
      "310\n",
      "311\n",
      "312\n",
      "313\n",
      "314\n",
      "315\n",
      "316\n",
      "317\n",
      "318\n",
      "319\n",
      "320\n",
      "321\n",
      "322\n",
      "323\n",
      "324\n",
      "325\n",
      "326\n",
      "327\n",
      "328\n",
      "329\n",
      "330\n",
      "331\n",
      "332\n",
      "333\n",
      "334\n",
      "335\n",
      "336\n",
      "337\n",
      "338\n",
      "339\n",
      "340\n",
      "341\n",
      "342\n",
      "343\n",
      "344\n",
      "345\n",
      "346\n",
      "347\n",
      "348\n",
      "349\n",
      "350\n",
      "351\n",
      "352\n",
      "353\n",
      "354\n",
      "355\n",
      "356\n",
      "357\n",
      "358\n",
      "359\n",
      "360\n",
      "361\n",
      "362\n",
      "363\n",
      "364\n",
      "365\n",
      "366\n",
      "367\n",
      "368\n",
      "369\n",
      "370\n",
      "371\n",
      "372\n",
      "373\n",
      "374\n",
      "375\n",
      "376\n",
      "377\n",
      "378\n",
      "379\n",
      "380\n",
      "381\n",
      "382\n",
      "383\n",
      "384\n",
      "385\n",
      "386\n",
      "387\n",
      "388\n",
      "389\n",
      "390\n",
      "391\n",
      "392\n",
      "393\n",
      "394\n",
      "395\n",
      "396\n",
      "397\n",
      "398\n",
      "399\n",
      "400\n",
      "401\n",
      "402\n",
      "403\n",
      "404\n",
      "405\n",
      "406\n",
      "407\n",
      "408\n",
      "409\n",
      "410\n",
      "411\n",
      "412\n",
      "413\n",
      "414\n",
      "415\n",
      "416\n",
      "417\n",
      "418\n",
      "419\n",
      "420\n",
      "421\n",
      "422\n",
      "423\n",
      "424\n",
      "425\n",
      "426\n",
      "427\n",
      "428\n",
      "429\n",
      "430\n",
      "431\n",
      "432\n",
      "433\n",
      "434\n",
      "435\n",
      "436\n",
      "437\n",
      "438\n",
      "439\n",
      "440\n",
      "441\n",
      "442\n",
      "443\n",
      "444\n",
      "445\n",
      "446\n",
      "447\n",
      "448\n",
      "449\n",
      "450\n",
      "451\n",
      "452\n",
      "453\n",
      "454\n",
      "455\n",
      "456\n",
      "457\n",
      "458\n",
      "459\n",
      "460\n",
      "461\n",
      "462\n",
      "463\n",
      "464\n",
      "465\n",
      "466\n",
      "467\n",
      "468\n",
      "469\n",
      "470\n",
      "471\n",
      "472\n",
      "473\n",
      "474\n",
      "475\n",
      "476\n",
      "477\n",
      "478\n",
      "479\n",
      "480\n",
      "481\n",
      "482\n",
      "483\n",
      "484\n",
      "485\n",
      "486\n",
      "487\n",
      "488\n",
      "489\n",
      "490\n",
      "491\n",
      "492\n",
      "493\n",
      "494\n",
      "495\n",
      "496\n",
      "497\n",
      "498\n",
      "499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:13:16 - INFO - __main__ -   ***** Running evaluation  *****\n",
      "03/26/2020 21:13:16 - INFO - __main__ -     Num examples = 2111\n",
      "03/26/2020 21:13:16 - INFO - __main__ -     Batch size = 16\n",
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:228: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd876a14c19a4bd8bd227001f1104e13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=132, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:13:22 - INFO - __main__ -   ***** Eval results  *****\n",
      "03/26/2020 21:13:22 - INFO - __main__ -     acc = 0.8943628612032212\n",
      "03/26/2020 21:13:22 - INFO - transformers.configuration_utils -   Configuration saved in ./outputs/checkpoint-500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:13:24 - INFO - transformers.modeling_utils -   Model weights saved in ./outputs/checkpoint-500/pytorch_model.bin\n",
      "03/26/2020 21:13:24 - INFO - __main__ -   Saving model checkpoint to ./outputs/checkpoint-500\n",
      "03/26/2020 21:13:29 - INFO - __main__ -   Saving optimizer and scheduler states to ./outputs/checkpoint-500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'last_epoch': 500, 'base_lrs': [5e-05, 5e-05], 'lr_lambdas': [None, None]}\n",
      "500\n",
      "501\n",
      "502\n",
      "503\n",
      "504\n",
      "505\n",
      "506\n",
      "507\n",
      "508\n",
      "509\n",
      "510\n",
      "511\n",
      "512\n",
      "513\n",
      "514\n",
      "515\n",
      "516\n",
      "517\n",
      "518\n",
      "519\n",
      "520\n",
      "521\n",
      "522\n",
      "523\n",
      "524\n",
      "525\n",
      "526\n",
      "527\n",
      "528\n",
      "529\n",
      "530\n",
      "531\n",
      "532\n",
      "533\n",
      "534\n",
      "535\n",
      "536\n",
      "537\n",
      "538\n",
      "539\n",
      "540\n",
      "541\n",
      "542\n",
      "543\n",
      "544\n",
      "545\n",
      "546\n",
      "547\n",
      "548\n",
      "549\n",
      "550\n",
      "551\n",
      "552\n",
      "553\n",
      "554\n",
      "555\n",
      "556\n",
      "557\n",
      "558\n",
      "559\n",
      "560\n",
      "561\n",
      "562\n",
      "563\n",
      "564\n",
      "565\n",
      "566\n",
      "567\n",
      "568\n",
      "569\n",
      "570\n",
      "571\n",
      "572\n",
      "573\n",
      "574\n",
      "575\n",
      "576\n",
      "577\n",
      "578\n",
      "579\n",
      "580\n",
      "581\n",
      "582\n",
      "583\n",
      "584\n",
      "585\n",
      "586\n",
      "587\n",
      "588\n",
      "589\n",
      "590\n",
      "591\n",
      "592\n",
      "593\n",
      "594\n",
      "595\n",
      "596\n",
      "597\n",
      "598\n",
      "599\n",
      "600\n",
      "601\n",
      "602\n",
      "603\n",
      "604\n",
      "605\n",
      "606\n",
      "607\n",
      "608\n",
      "609\n",
      "610\n",
      "611\n",
      "612\n",
      "613\n",
      "614\n",
      "615\n",
      "616\n",
      "617\n",
      "618\n",
      "619\n",
      "620\n",
      "621\n",
      "622\n",
      "623\n",
      "624\n",
      "625\n",
      "626\n",
      "627\n",
      "628\n",
      "629\n",
      "630\n",
      "631\n",
      "632\n",
      "633\n",
      "634\n",
      "635\n",
      "636\n",
      "637\n",
      "638\n",
      "639\n",
      "640\n",
      "641\n",
      "642\n",
      "643\n",
      "644\n",
      "645\n",
      "646\n",
      "647\n",
      "648\n",
      "649\n",
      "650\n",
      "651\n",
      "652\n",
      "653\n",
      "654\n",
      "655\n",
      "656\n",
      "657\n",
      "658\n",
      "659\n",
      "660\n",
      "661\n",
      "662\n",
      "663\n",
      "664\n",
      "665\n",
      "666\n",
      "667\n",
      "668\n",
      "669\n",
      "670\n",
      "671\n",
      "672\n",
      "673\n",
      "674\n",
      "675\n",
      "676\n",
      "677\n",
      "678\n",
      "679\n",
      "680\n",
      "681\n",
      "682\n",
      "683\n",
      "684\n",
      "685\n",
      "686\n",
      "687\n",
      "688\n",
      "689\n",
      "690\n",
      "691\n",
      "692\n",
      "693\n",
      "694\n",
      "695\n",
      "696\n",
      "697\n",
      "698\n",
      "699\n",
      "700\n",
      "701\n",
      "702\n",
      "703\n",
      "704\n",
      "705\n",
      "706\n",
      "707\n",
      "708\n",
      "709\n",
      "710\n",
      "711\n",
      "712\n",
      "713\n",
      "714\n",
      "715\n",
      "716\n",
      "717\n",
      "718\n",
      "719\n",
      "720\n",
      "721\n",
      "722\n",
      "723\n",
      "724\n",
      "725\n",
      "726\n",
      "727\n",
      "728\n",
      "729\n",
      "730\n",
      "731\n",
      "732\n",
      "733\n",
      "734\n",
      "735\n",
      "736\n",
      "737\n",
      "738\n",
      "739\n",
      "740\n",
      "741\n",
      "742\n",
      "743\n",
      "744\n",
      "745\n",
      "746\n",
      "747\n",
      "748\n",
      "749\n",
      "750\n",
      "751\n",
      "752\n",
      "753\n",
      "754\n",
      "755\n",
      "756\n",
      "757\n",
      "758\n",
      "759\n",
      "760\n",
      "761\n",
      "762\n",
      "763\n",
      "764\n",
      "765\n",
      "766\n",
      "767\n",
      "768\n",
      "769\n",
      "770\n",
      "771\n",
      "772\n",
      "773\n",
      "774\n",
      "775\n",
      "776\n",
      "777\n",
      "778\n",
      "779\n",
      "780\n",
      "781\n",
      "782\n",
      "783\n",
      "784\n",
      "785\n",
      "786\n",
      "787\n",
      "788\n",
      "789\n",
      "790\n",
      "791\n",
      "792\n",
      "793\n",
      "794\n",
      "795\n",
      "796\n",
      "797\n",
      "798\n",
      "799\n",
      "800\n",
      "801\n",
      "802\n",
      "803\n",
      "804\n",
      "805\n",
      "806\n",
      "807\n",
      "808\n",
      "809\n",
      "810\n",
      "811\n",
      "812\n",
      "813\n",
      "814\n",
      "815\n",
      "816\n",
      "817\n",
      "818\n",
      "819\n",
      "820\n",
      "821\n",
      "822\n",
      "823\n",
      "824\n",
      "825\n",
      "826\n",
      "827\n",
      "828\n",
      "829\n",
      "830\n",
      "831\n",
      "832\n",
      "833\n",
      "834\n",
      "835\n",
      "836\n",
      "837\n",
      "838\n",
      "839\n",
      "840\n",
      "841\n",
      "842\n",
      "843\n",
      "844\n",
      "845\n",
      "846\n",
      "847\n",
      "848\n",
      "849\n",
      "850\n",
      "851\n",
      "852\n",
      "853\n",
      "854\n",
      "855\n",
      "856\n",
      "857\n",
      "858\n",
      "859\n",
      "860\n",
      "861\n",
      "862\n",
      "863\n",
      "864\n",
      "865\n",
      "866\n",
      "867\n",
      "868\n",
      "869\n",
      "870\n",
      "871\n",
      "872\n",
      "873\n",
      "874\n",
      "875\n",
      "876\n",
      "877\n",
      "878\n",
      "879\n",
      "880\n",
      "881\n",
      "882\n",
      "883\n",
      "884\n",
      "885\n",
      "886\n",
      "887\n",
      "888\n",
      "889\n",
      "890\n",
      "891\n",
      "892\n",
      "893\n",
      "894\n",
      "895\n",
      "896\n",
      "897\n",
      "898\n",
      "899\n",
      "900\n",
      "901\n",
      "902\n",
      "903\n",
      "904\n",
      "905\n",
      "906\n",
      "907\n",
      "908\n",
      "909\n",
      "910\n",
      "911\n",
      "912\n",
      "913\n",
      "914\n",
      "915\n",
      "916\n",
      "917\n",
      "918\n",
      "919\n",
      "920\n",
      "921\n",
      "922\n",
      "923\n",
      "924\n",
      "925\n",
      "926\n",
      "927\n",
      "928\n",
      "929\n",
      "930\n",
      "931\n",
      "932\n",
      "933\n",
      "934\n",
      "935\n",
      "936\n",
      "937\n",
      "938\n",
      "939\n",
      "940\n",
      "941\n",
      "942\n",
      "943\n",
      "944\n",
      "945\n",
      "946\n",
      "947\n",
      "948\n",
      "949\n",
      "950\n",
      "951\n",
      "952\n",
      "953\n",
      "954\n",
      "955\n",
      "956\n",
      "957\n",
      "958\n",
      "959\n",
      "960\n",
      "961\n",
      "962\n",
      "963\n",
      "964\n",
      "965\n",
      "966\n",
      "967\n",
      "968\n",
      "969\n",
      "970\n",
      "971\n",
      "972\n",
      "973\n",
      "974\n",
      "975\n",
      "976\n",
      "977\n",
      "978\n",
      "979\n",
      "980\n",
      "981\n",
      "982\n",
      "983\n",
      "984\n",
      "985\n",
      "986\n",
      "987\n",
      "988\n",
      "989\n",
      "990\n",
      "991\n",
      "992\n",
      "993\n",
      "994\n",
      "995\n",
      "996\n",
      "997\n",
      "998\n",
      "999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:14:59 - INFO - __main__ -   ***** Running evaluation  *****\n",
      "03/26/2020 21:14:59 - INFO - __main__ -     Num examples = 2111\n",
      "03/26/2020 21:14:59 - INFO - __main__ -     Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a48fc6208e0443608a96ab9c3d8f5348",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=132, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:15:06 - INFO - __main__ -   ***** Eval results  *****\n",
      "03/26/2020 21:15:06 - INFO - __main__ -     acc = 0.9317858834675509\n",
      "03/26/2020 21:15:06 - INFO - transformers.configuration_utils -   Configuration saved in ./outputs/checkpoint-1000/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:15:08 - INFO - transformers.modeling_utils -   Model weights saved in ./outputs/checkpoint-1000/pytorch_model.bin\n",
      "03/26/2020 21:15:08 - INFO - __main__ -   Saving model checkpoint to ./outputs/checkpoint-1000\n",
      "03/26/2020 21:15:14 - INFO - __main__ -   Saving optimizer and scheduler states to ./outputs/checkpoint-1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'last_epoch': 1000, 'base_lrs': [5e-05, 5e-05], 'lr_lambdas': [None, None]}\n",
      "1000\n",
      "1001\n",
      "1002\n",
      "1003\n",
      "1004\n",
      "1005\n",
      "1006\n",
      "1007\n",
      "1008\n",
      "1009\n",
      "1010\n",
      "1011\n",
      "1012\n",
      "1013\n",
      "1014\n",
      "1015\n",
      "1016\n",
      "1017\n",
      "1018\n",
      "1019\n",
      "1020\n",
      "1021\n",
      "1022\n",
      "1023\n",
      "1024\n",
      "1025\n",
      "1026\n",
      "1027\n",
      "1028\n",
      "1029\n",
      "1030\n",
      "1031\n",
      "1032\n",
      "1033\n",
      "1034\n",
      "1035\n",
      "1036\n",
      "1037\n",
      "1038\n",
      "1039\n",
      "1040\n",
      "1041\n",
      "1042\n",
      "1043\n",
      "1044\n",
      "1045\n",
      "1046\n",
      "1047\n",
      "1048\n",
      "1049\n",
      "1050\n",
      "1051\n",
      "1052\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b20387b2ee774241ba8eebce760d157f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=1053, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n",
      "210\n",
      "211\n",
      "212\n",
      "213\n",
      "214\n",
      "215\n",
      "216\n",
      "217\n",
      "218\n",
      "219\n",
      "220\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "234\n",
      "235\n",
      "236\n",
      "237\n",
      "238\n",
      "239\n",
      "240\n",
      "241\n",
      "242\n",
      "243\n",
      "244\n",
      "245\n",
      "246\n",
      "247\n",
      "248\n",
      "249\n",
      "250\n",
      "251\n",
      "252\n",
      "253\n",
      "254\n",
      "255\n",
      "256\n",
      "257\n",
      "258\n",
      "259\n",
      "260\n",
      "261\n",
      "262\n",
      "263\n",
      "264\n",
      "265\n",
      "266\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "287\n",
      "288\n",
      "289\n",
      "290\n",
      "291\n",
      "292\n",
      "293\n",
      "294\n",
      "295\n",
      "296\n",
      "297\n",
      "298\n",
      "299\n",
      "300\n",
      "301\n",
      "302\n",
      "303\n",
      "304\n",
      "305\n",
      "306\n",
      "307\n",
      "308\n",
      "309\n",
      "310\n",
      "311\n",
      "312\n",
      "313\n",
      "314\n",
      "315\n",
      "316\n",
      "317\n",
      "318\n",
      "319\n",
      "320\n",
      "321\n",
      "322\n",
      "323\n",
      "324\n",
      "325\n",
      "326\n",
      "327\n",
      "328\n",
      "329\n",
      "330\n",
      "331\n",
      "332\n",
      "333\n",
      "334\n",
      "335\n",
      "336\n",
      "337\n",
      "338\n",
      "339\n",
      "340\n",
      "341\n",
      "342\n",
      "343\n",
      "344\n",
      "345\n",
      "346\n",
      "347\n",
      "348\n",
      "349\n",
      "350\n",
      "351\n",
      "352\n",
      "353\n",
      "354\n",
      "355\n",
      "356\n",
      "357\n",
      "358\n",
      "359\n",
      "360\n",
      "361\n",
      "362\n",
      "363\n",
      "364\n",
      "365\n",
      "366\n",
      "367\n",
      "368\n",
      "369\n",
      "370\n",
      "371\n",
      "372\n",
      "373\n",
      "374\n",
      "375\n",
      "376\n",
      "377\n",
      "378\n",
      "379\n",
      "380\n",
      "381\n",
      "382\n",
      "383\n",
      "384\n",
      "385\n",
      "386\n",
      "387\n",
      "388\n",
      "389\n",
      "390\n",
      "391\n",
      "392\n",
      "393\n",
      "394\n",
      "395\n",
      "396\n",
      "397\n",
      "398\n",
      "399\n",
      "400\n",
      "401\n",
      "402\n",
      "403\n",
      "404\n",
      "405\n",
      "406\n",
      "407\n",
      "408\n",
      "409\n",
      "410\n",
      "411\n",
      "412\n",
      "413\n",
      "414\n",
      "415\n",
      "416\n",
      "417\n",
      "418\n",
      "419\n",
      "420\n",
      "421\n",
      "422\n",
      "423\n",
      "424\n",
      "425\n",
      "426\n",
      "427\n",
      "428\n",
      "429\n",
      "430\n",
      "431\n",
      "432\n",
      "433\n",
      "434\n",
      "435\n",
      "436\n",
      "437\n",
      "438\n",
      "439\n",
      "440\n",
      "441\n",
      "442\n",
      "443\n",
      "444\n",
      "445\n",
      "446\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:16:48 - INFO - __main__ -   ***** Running evaluation  *****\n",
      "03/26/2020 21:16:48 - INFO - __main__ -     Num examples = 2111\n",
      "03/26/2020 21:16:48 - INFO - __main__ -     Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "919f26d16267400db6f222251e5aca20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=132, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:16:55 - INFO - __main__ -   ***** Eval results  *****\n",
      "03/26/2020 21:16:55 - INFO - __main__ -     acc = 0.9369966840360019\n",
      "03/26/2020 21:16:55 - INFO - transformers.configuration_utils -   Configuration saved in ./outputs/checkpoint-1500/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:16:57 - INFO - transformers.modeling_utils -   Model weights saved in ./outputs/checkpoint-1500/pytorch_model.bin\n",
      "03/26/2020 21:16:57 - INFO - __main__ -   Saving model checkpoint to ./outputs/checkpoint-1500\n",
      "03/26/2020 21:17:02 - INFO - __main__ -   Saving optimizer and scheduler states to ./outputs/checkpoint-1500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'last_epoch': 1500, 'base_lrs': [5e-05, 5e-05], 'lr_lambdas': [None, None]}\n",
      "447\n",
      "448\n",
      "449\n",
      "450\n",
      "451\n",
      "452\n",
      "453\n",
      "454\n",
      "455\n",
      "456\n",
      "457\n",
      "458\n",
      "459\n",
      "460\n",
      "461\n",
      "462\n",
      "463\n",
      "464\n",
      "465\n",
      "466\n",
      "467\n",
      "468\n",
      "469\n",
      "470\n",
      "471\n",
      "472\n",
      "473\n",
      "474\n",
      "475\n",
      "476\n",
      "477\n",
      "478\n",
      "479\n",
      "480\n",
      "481\n",
      "482\n",
      "483\n",
      "484\n",
      "485\n",
      "486\n",
      "487\n",
      "488\n",
      "489\n",
      "490\n",
      "491\n",
      "492\n",
      "493\n",
      "494\n",
      "495\n",
      "496\n",
      "497\n",
      "498\n",
      "499\n",
      "500\n",
      "501\n",
      "502\n",
      "503\n",
      "504\n",
      "505\n",
      "506\n",
      "507\n",
      "508\n",
      "509\n",
      "510\n",
      "511\n",
      "512\n",
      "513\n",
      "514\n",
      "515\n",
      "516\n",
      "517\n",
      "518\n",
      "519\n",
      "520\n",
      "521\n",
      "522\n",
      "523\n",
      "524\n",
      "525\n",
      "526\n",
      "527\n",
      "528\n",
      "529\n",
      "530\n",
      "531\n",
      "532\n",
      "533\n",
      "534\n",
      "535\n",
      "536\n",
      "537\n",
      "538\n",
      "539\n",
      "540\n",
      "541\n",
      "542\n",
      "543\n",
      "544\n",
      "545\n",
      "546\n",
      "547\n",
      "548\n",
      "549\n",
      "550\n",
      "551\n",
      "552\n",
      "553\n",
      "554\n",
      "555\n",
      "556\n",
      "557\n",
      "558\n",
      "559\n",
      "560\n",
      "561\n",
      "562\n",
      "563\n",
      "564\n",
      "565\n",
      "566\n",
      "567\n",
      "568\n",
      "569\n",
      "570\n",
      "571\n",
      "572\n",
      "573\n",
      "574\n",
      "575\n",
      "576\n",
      "577\n",
      "578\n",
      "579\n",
      "580\n",
      "581\n",
      "582\n",
      "583\n",
      "584\n",
      "585\n",
      "586\n",
      "587\n",
      "588\n",
      "589\n",
      "590\n",
      "591\n",
      "592\n",
      "593\n",
      "594\n",
      "595\n",
      "596\n",
      "597\n",
      "598\n",
      "599\n",
      "600\n",
      "601\n",
      "602\n",
      "603\n",
      "604\n",
      "605\n",
      "606\n",
      "607\n",
      "608\n",
      "609\n",
      "610\n",
      "611\n",
      "612\n",
      "613\n",
      "614\n",
      "615\n",
      "616\n",
      "617\n",
      "618\n",
      "619\n",
      "620\n",
      "621\n",
      "622\n",
      "623\n",
      "624\n",
      "625\n",
      "626\n",
      "627\n",
      "628\n",
      "629\n",
      "630\n",
      "631\n",
      "632\n",
      "633\n",
      "634\n",
      "635\n",
      "636\n",
      "637\n",
      "638\n",
      "639\n",
      "640\n",
      "641\n",
      "642\n",
      "643\n",
      "644\n",
      "645\n",
      "646\n",
      "647\n",
      "648\n",
      "649\n",
      "650\n",
      "651\n",
      "652\n",
      "653\n",
      "654\n",
      "655\n",
      "656\n",
      "657\n",
      "658\n",
      "659\n",
      "660\n",
      "661\n",
      "662\n",
      "663\n",
      "664\n",
      "665\n",
      "666\n",
      "667\n",
      "668\n",
      "669\n",
      "670\n",
      "671\n",
      "672\n",
      "673\n",
      "674\n",
      "675\n",
      "676\n",
      "677\n",
      "678\n",
      "679\n",
      "680\n",
      "681\n",
      "682\n",
      "683\n",
      "684\n",
      "685\n",
      "686\n",
      "687\n",
      "688\n",
      "689\n",
      "690\n",
      "691\n",
      "692\n",
      "693\n",
      "694\n",
      "695\n",
      "696\n",
      "697\n",
      "698\n",
      "699\n",
      "700\n",
      "701\n",
      "702\n",
      "703\n",
      "704\n",
      "705\n",
      "706\n",
      "707\n",
      "708\n",
      "709\n",
      "710\n",
      "711\n",
      "712\n",
      "713\n",
      "714\n",
      "715\n",
      "716\n",
      "717\n",
      "718\n",
      "719\n",
      "720\n",
      "721\n",
      "722\n",
      "723\n",
      "724\n",
      "725\n",
      "726\n",
      "727\n",
      "728\n",
      "729\n",
      "730\n",
      "731\n",
      "732\n",
      "733\n",
      "734\n",
      "735\n",
      "736\n",
      "737\n",
      "738\n",
      "739\n",
      "740\n",
      "741\n",
      "742\n",
      "743\n",
      "744\n",
      "745\n",
      "746\n",
      "747\n",
      "748\n",
      "749\n",
      "750\n",
      "751\n",
      "752\n",
      "753\n",
      "754\n",
      "755\n",
      "756\n",
      "757\n",
      "758\n",
      "759\n",
      "760\n",
      "761\n",
      "762\n",
      "763\n",
      "764\n",
      "765\n",
      "766\n",
      "767\n",
      "768\n",
      "769\n",
      "770\n",
      "771\n",
      "772\n",
      "773\n",
      "774\n",
      "775\n",
      "776\n",
      "777\n",
      "778\n",
      "779\n",
      "780\n",
      "781\n",
      "782\n",
      "783\n",
      "784\n",
      "785\n",
      "786\n",
      "787\n",
      "788\n",
      "789\n",
      "790\n",
      "791\n",
      "792\n",
      "793\n",
      "794\n",
      "795\n",
      "796\n",
      "797\n",
      "798\n",
      "799\n",
      "800\n",
      "801\n",
      "802\n",
      "803\n",
      "804\n",
      "805\n",
      "806\n",
      "807\n",
      "808\n",
      "809\n",
      "810\n",
      "811\n",
      "812\n",
      "813\n",
      "814\n",
      "815\n",
      "816\n",
      "817\n",
      "818\n",
      "819\n",
      "820\n",
      "821\n",
      "822\n",
      "823\n",
      "824\n",
      "825\n",
      "826\n",
      "827\n",
      "828\n",
      "829\n",
      "830\n",
      "831\n",
      "832\n",
      "833\n",
      "834\n",
      "835\n",
      "836\n",
      "837\n",
      "838\n",
      "839\n",
      "840\n",
      "841\n",
      "842\n",
      "843\n",
      "844\n",
      "845\n",
      "846\n",
      "847\n",
      "848\n",
      "849\n",
      "850\n",
      "851\n",
      "852\n",
      "853\n",
      "854\n",
      "855\n",
      "856\n",
      "857\n",
      "858\n",
      "859\n",
      "860\n",
      "861\n",
      "862\n",
      "863\n",
      "864\n",
      "865\n",
      "866\n",
      "867\n",
      "868\n",
      "869\n",
      "870\n",
      "871\n",
      "872\n",
      "873\n",
      "874\n",
      "875\n",
      "876\n",
      "877\n",
      "878\n",
      "879\n",
      "880\n",
      "881\n",
      "882\n",
      "883\n",
      "884\n",
      "885\n",
      "886\n",
      "887\n",
      "888\n",
      "889\n",
      "890\n",
      "891\n",
      "892\n",
      "893\n",
      "894\n",
      "895\n",
      "896\n",
      "897\n",
      "898\n",
      "899\n",
      "900\n",
      "901\n",
      "902\n",
      "903\n",
      "904\n",
      "905\n",
      "906\n",
      "907\n",
      "908\n",
      "909\n",
      "910\n",
      "911\n",
      "912\n",
      "913\n",
      "914\n",
      "915\n",
      "916\n",
      "917\n",
      "918\n",
      "919\n",
      "920\n",
      "921\n",
      "922\n",
      "923\n",
      "924\n",
      "925\n",
      "926\n",
      "927\n",
      "928\n",
      "929\n",
      "930\n",
      "931\n",
      "932\n",
      "933\n",
      "934\n",
      "935\n",
      "936\n",
      "937\n",
      "938\n",
      "939\n",
      "940\n",
      "941\n",
      "942\n",
      "943\n",
      "944\n",
      "945\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:18:39 - INFO - __main__ -   ***** Running evaluation  *****\n",
      "03/26/2020 21:18:39 - INFO - __main__ -     Num examples = 2111\n",
      "03/26/2020 21:18:39 - INFO - __main__ -     Batch size = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "946\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd52ab9c85ac4de8a18b8e45a5819856",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=132, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:18:46 - INFO - __main__ -   ***** Eval results  *****\n",
      "03/26/2020 21:18:46 - INFO - __main__ -     acc = 0.9436286120322123\n",
      "03/26/2020 21:18:46 - INFO - transformers.configuration_utils -   Configuration saved in ./outputs/checkpoint-2000/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:18:48 - INFO - transformers.modeling_utils -   Model weights saved in ./outputs/checkpoint-2000/pytorch_model.bin\n",
      "03/26/2020 21:18:48 - INFO - __main__ -   Saving model checkpoint to ./outputs/checkpoint-2000\n",
      "03/26/2020 21:18:53 - INFO - __main__ -   Saving optimizer and scheduler states to ./outputs/checkpoint-2000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'last_epoch': 2000, 'base_lrs': [5e-05, 5e-05], 'lr_lambdas': [None, None]}\n",
      "947\n",
      "948\n",
      "949\n",
      "950\n",
      "951\n",
      "952\n",
      "953\n",
      "954\n",
      "955\n",
      "956\n",
      "957\n",
      "958\n",
      "959\n",
      "960\n",
      "961\n",
      "962\n",
      "963\n",
      "964\n",
      "965\n",
      "966\n",
      "967\n",
      "968\n",
      "969\n",
      "970\n",
      "971\n",
      "972\n",
      "973\n",
      "974\n",
      "975\n",
      "976\n",
      "977\n",
      "978\n",
      "979\n",
      "980\n",
      "981\n",
      "982\n",
      "983\n",
      "984\n",
      "985\n",
      "986\n",
      "987\n",
      "988\n",
      "989\n",
      "990\n",
      "991\n",
      "992\n",
      "993\n",
      "994\n",
      "995\n",
      "996\n",
      "997\n",
      "998\n",
      "999\n",
      "1000\n",
      "1001\n",
      "1002\n",
      "1003\n",
      "1004\n",
      "1005\n",
      "1006\n",
      "1007\n",
      "1008\n",
      "1009\n",
      "1010\n",
      "1011\n",
      "1012\n",
      "1013\n",
      "1014\n",
      "1015\n",
      "1016\n",
      "1017\n",
      "1018\n",
      "1019\n",
      "1020\n",
      "1021\n",
      "1022\n",
      "1023\n",
      "1024\n",
      "1025\n",
      "1026\n",
      "1027\n",
      "1028\n",
      "1029\n",
      "1030\n",
      "1031\n",
      "1032\n",
      "1033\n",
      "1034\n",
      "1035\n",
      "1036\n",
      "1037\n",
      "1038\n",
      "1039\n",
      "1040\n",
      "1041\n",
      "1042\n",
      "1043\n",
      "1044\n",
      "1045\n",
      "1046\n",
      "1047\n",
      "1048\n",
      "1049\n",
      "1050\n",
      "1051\n",
      "1052\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "set_seed(args)\n",
    "model.to(args.device)\n",
    "global_step, tr_loss = train(args,sentiment_train_set,sentiment_valid_set, model, tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### 二次加载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/26/2020 21:06:32 - INFO - transformers.configuration_utils -   loading configuration file ./outputs/checkpoint-2000/config.json\n",
      "03/26/2020 21:06:32 - INFO - transformers.configuration_utils -   Model config BertConfig {\n",
      "  \"_num_labels\": 2,\n",
      "  \"architectures\": [\n",
      "    \"BertFJW\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": null,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"do_sample\": false,\n",
      "  \"early_stopping\": false,\n",
      "  \"eos_token_ids\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.3,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": true,\n",
      "  \"output_hidden_states\": true,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": null,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "03/26/2020 21:06:32 - INFO - transformers.modeling_utils -   loading weights file ./outputs/checkpoint-2000/pytorch_model.bin\n",
      "03/26/2020 21:06:35 - INFO - transformers.tokenization_utils -   Model name './outputs/checkpoint-2000/' not found in model shortcut name list (bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese, bert-base-german-cased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking, bert-large-uncased-whole-word-masking-finetuned-squad, bert-large-cased-whole-word-masking-finetuned-squad, bert-base-cased-finetuned-mrpc, bert-base-german-dbmdz-cased, bert-base-german-dbmdz-uncased, bert-base-finnish-cased-v1, bert-base-finnish-uncased-v1, bert-base-dutch-cased). Assuming './outputs/checkpoint-2000/' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "03/26/2020 21:06:35 - INFO - transformers.tokenization_utils -   Didn't find file ./outputs/checkpoint-2000/added_tokens.json. We won't load it.\n",
      "03/26/2020 21:06:35 - INFO - transformers.tokenization_utils -   loading file ./outputs/checkpoint-2000/vocab.txt\n",
      "03/26/2020 21:06:35 - INFO - transformers.tokenization_utils -   loading file None\n",
      "03/26/2020 21:06:35 - INFO - transformers.tokenization_utils -   loading file ./outputs/checkpoint-2000/special_tokens_map.json\n",
      "03/26/2020 21:06:35 - INFO - transformers.tokenization_utils -   loading file ./outputs/checkpoint-2000/tokenizer_config.json\n"
     ]
    }
   ],
   "source": [
    "model=BertFJW.from_pretrained(\"./outputs/checkpoint-2000/\")\n",
    "tokenizer=BertTokenizer.from_pretrained(\"./outputs/checkpoint-2000/\")\n",
    "args=torch.load(\"./outputs/checkpoint-2000/training_args.bin\")\n",
    "train_params=args._asdict()\n",
    "train_params[\"model_name_or_path\"]=\"./outputs/checkpoint-2000/\"\n",
    "train_params[\"num_train_epochs\"]+=2\n",
    "args=ARG(**train_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# set_seed(args)\n",
    "# model.to(args.device)\n",
    "# global_step, tr_loss = train(args,sentiment_train_set,sentiment_valid_set, model, tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## 我的尝试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Calling BertTokenizer.from_pretrained() with the path to a single file or url is deprecated\n"
     ]
    }
   ],
   "source": [
    "hidden_dropout_prob = 0.3\n",
    "num_labels = 2\n",
    "learning_rate = 1e-5\n",
    "weight_decay = 1e-2\n",
    "epochs = 2\n",
    "batch_size = 16\n",
    "device=torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
    "bert_dir=\"./bert-pytorch-chinese/\"\n",
    "vocab=\"bert-base-chinese-vocab.txt\"\n",
    "config_file=\"bert_config.json\"\n",
    "config=BertConfig.from_pretrained(os.path.join(bert_dir,config_file),output_hidden_states=True,\n",
    "                                        output_attentions=True,hidden_dropout_prob=hidden_dropout_prob)\n",
    "model=BertModel.from_pretrained(bert_dir,config=config)\n",
    "tokenizer=BertTokenizer.from_pretrained(os.path.join(bert_dir,vocab))\n",
    "model=model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "code_folding": [
     0,
     16,
     25
    ],
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def convert_text_to_ids(tokenizer, text, max_len=100):\n",
    "    if isinstance(text,str):\n",
    "        output=tokenizer.encode_plus(text,max_length=max_len,pad_to_max_length=True,return_tensors=\"pt\")\n",
    "        input_ids=output[\"input_ids\"].squeeze(0)\n",
    "        token_type_ids=output[\"token_type_ids\"].squeeze(0)\n",
    "        attention_mask=output[\"attention_mask\"].squeeze(0)\n",
    "    elif isinstance(text,list):\n",
    "        input_ids,token_type_ids,attention_mask=[],[],[]\n",
    "        for e in text:\n",
    "            output=tokenizer.encode_plus(e,max_length=max_len,pad_to_max_length=True,return_tensors=\"pt\")\n",
    "            input_ids.append(output[\"input_ids\"].squeeze(0))\n",
    "            token_type_ids.append(output[\"token_type_ids\"].squeeze(0))\n",
    "            attention_mask.append(output[\"attention_mask\"].squeeze(0))\n",
    "    else:\n",
    "        raise Exception('type error')\n",
    "    return torch.stack(input_ids).long(),torch.stack(token_type_ids).long(),torch.stack(attention_mask).long()        \n",
    "class SentimentDataset(Data.Dataset):\n",
    "    def __init__(self,path):\n",
    "        self.dataset=pd.read_csv(path,sep=\"\\t\",names=[\"text\",\"label\"])\n",
    "    def __len__(self):\n",
    "        return self.dataset.shape[0]\n",
    "    def __getitem__(self,idx):\n",
    "        text=self.dataset.loc[idx,\"text\"]\n",
    "        label=self.dataset.loc[idx,\"label\"]\n",
    "        return {\"text\":text,\"label\":label}\n",
    "class BertClassify(nn.Module):\n",
    "    def __init__(self,model,num_labels,dropout_rate):\n",
    "        super(BertClassify,self).__init__()\n",
    "        self.bert=model\n",
    "        self.Dense=nn.Sequential(nn.Dropout(dropout_rate),nn.Linear(model.pooler.dense.out_features,num_labels))\n",
    "    def forward(self,X,attention_mask,token_type_ids):\n",
    "        pooled_output=self.bert(X,attention_mask=attention_mask,token_type_ids=token_type_ids)[1]\n",
    "        return self.Dense(pooled_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "data_path = \"./test-data/sentiment/\"\n",
    "# 加载数据集\n",
    "sentiment_train_set = SentimentDataset(data_path + \"sentiment.train.data\")\n",
    "sentiment_train_loader =Data.DataLoader(sentiment_train_set, batch_size=batch_size, shuffle=True, num_workers=2)\n",
    "\n",
    "sentiment_valid_set = SentimentDataset(data_path + \"sentiment.valid.data\")\n",
    "sentiment_valid_loader = Data.DataLoader(sentiment_valid_set, batch_size=batch_size, shuffle=False, num_workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model=BertClassify(model,num_labels,0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "no_decay = ['bias', 'LayerNorm.weight']\n",
    "optim_group_params=[{'params':[ p for n,p in model.named_parameters() if not any(np in n for np in no_decay)],\"weight_decay\":weight_decay},\n",
    "                   {'params':[ p for n,p in model.named_parameters() if  any(np in n for np in no_decay)],\"weight_decay\":0}]\n",
    "optimizer=AdamW(optim_group_params,lr=learning_rate)\n",
    "loss=nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "code_folding": [
     0,
     27
    ],
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def train(model, train_iter,test_iter, optimizer, loss, device,num_epochs):\n",
    "    model=model.to(device)\n",
    "    for epoch  in range(num_epochs):\n",
    "        model.train()\n",
    "        epoch_loss=0\n",
    "        epoch_acc=0.\n",
    "        for i,batch in enumerate(train_iter):\n",
    "            text=batch[\"text\"]\n",
    "            label=batch[\"label\"]\n",
    "            input_ids,token_type_ids,attention_mask=convert_text_to_ids(tokenizer,text)\n",
    "            label,input_ids,token_type_ids,attention_mask=label.to(device),input_ids.to(device),token_type_ids.to(device),attention_mask.to(device)\n",
    "            outputs=model(input_ids,attention_mask=attention_mask,token_type_ids=token_type_ids)\n",
    "            label_pred_probs=outputs\n",
    "            label_pred=label_pred_probs.argmax(dim=-1)\n",
    "            optimizer.zero_grad()\n",
    "            l=loss(label_pred_probs,label)\n",
    "            acc=(label_pred==label).sum().cpu().item()\n",
    "            l.backward()\n",
    "            optimizer.step()\n",
    "            epoch_acc+=acc\n",
    "            epoch_loss+=l.cpu().item()\n",
    "            if i % 200 == 0:\n",
    "                print(\"current loss:\", epoch_loss / (i+1), \"\\t\", \"current acc:\", epoch_acc / ((i+1)*len(label)))\n",
    "        test_loss,test_acc=evaluate(model,test_iter,loss,device)\n",
    "        print(\"train epoch loss:%.3f epoch acc:%.3f\"%(epoch_loss / len(train_iter), epoch_acc / len(train_iter.dataset.dataset)))\n",
    "        print(\"test epoch loss:%.3f epoch acc:%.3f\"%(test_loss, test_acc))\n",
    "\n",
    "def evaluate(model, iterator, loss, device):\n",
    "    model=model.to(device)\n",
    "    model.eval()\n",
    "    epoch_loss=0\n",
    "    epoch_acc=0.\n",
    "    for i,batch in enumerate(iterator):\n",
    "        text=batch[\"text\"]\n",
    "        label=batch[\"label\"]\n",
    "        input_ids,token_type_ids,attention_mask=convert_text_to_ids(tokenizer,text)\n",
    "        label,input_ids,token_type_ids,attention_mask=label.to(device),input_ids.to(device),token_type_ids.to(device),attention_mask.to(device)\n",
    "        outputs=model(input_ids,attention_mask=attention_mask,token_type_ids=token_type_ids)\n",
    "        label_pred_probs=outputs\n",
    "        label_pred=label_pred_probs.argmax(dim=-1)\n",
    "        l=loss(label_pred_probs,label)\n",
    "        acc=(label_pred==label).sum().cpu().item()\n",
    "        epoch_acc+=acc\n",
    "        epoch_loss+=l.cpu().item()\n",
    "    model.train()\n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator.dataset.dataset)       \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train(model,sentiment_train_loader,sentiment_valid_loader,optimizer,loss,device,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/torch/serialization.py:256: UserWarning: Couldn't retrieve source code for container of type BertClassify. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "torch.save(model,\"./model.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model=torch.load(\"./model.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.11428531620683324, 0.9593800106894709)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model,sentiment_valid_loader,loss,device)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fjw",
   "language": "python",
   "name": "fjw"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
